#!/usr/bin/env python
# coding: utf-8

import pandas as pd
import re
import sys
from unicodedata import category as cat

def zh_stopwords_list(txt_path):
    """
    将中文停止词转换为列表
    txt_path为txt格式的中文停止词路径
    """
    zh_stopwords = pd.read_csv(txt_path, encoding="utf8", header=None)
    stopwords = list(zh_stopwords[0].values)
    return stopwords

def strQ2B(ustring):
    """把字串全形转半形"""
    ss = []
    for s in ustring:
        rstring = ""
        for uchar in s:
            inside_code = ord(uchar)
            if inside_code == 12288:  # 全形空格直接转换
                inside_code = 32
            elif (inside_code >= 65281 and inside_code <= 65374):  # 全形字元（除空格）根据关系转换
                inside_code -= 65248
            rstring += chr(inside_code)
        ss.append(rstring)
    return ''.join(ss)


def submissions_doc(df_meta):
    """
    整合原表特定栏位，过滤掉标点
    """
    df = df_meta
    tbl_p = [chr(i) for i in range(sys.maxunicode) if cat(chr(i)).startswith('P')]
    # 过滤.maketrans
    intab = "".join(tbl_p)+"︱"
    outtab = " " * len(intab)
    trantab = "".maketrans(intab, outtab)  # maketrans 创建字符映射的转换表
    sub_punc_w_ws = lambda x: x.translate(trantab)
    df[["doc"]].applymap(sub_punc_w_ws)
    dfc = df.copy()
    up = df[["doc"]].applymap(sub_punc_w_ws)
    dfc.update(up)
    sub_ws = lambda x: re.sub( r'\s+', " ", x)## preprocessing, removing whitespaces(multiple) to 1 
    dfc["doc_cleaned"] = dfc["doc"].map(sub_ws)
    dfc["doc_cleaned"] = dfc["doc_cleaned"].map(strQ2B)
    # dfc['BOW_JB'] = [ws_jieba(  df.doc[i]  ) for i in dfc.index]
    return dfc






